{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import Ridge, Lasso\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import warnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_college = pd.read_csv('college.csv')\n",
    "df_college.isna().any().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_list = df_college.columns.to_list()\n",
    "x_list.remove('Apps')\n",
    "x_list.remove('Private')\n",
    "x = df_college[x_list]\n",
    "y = df_college['Apps']\n",
    "x_std = (x - x.mean())/y.std(0)\n",
    "y_std = y - y.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)\n",
    "x_train_std, x_test_std, y_train_std, y_test_std = train_test_split(x_std, y_std, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def repeat_test(ridge_true, std_true):\n",
    "    warnings.filterwarnings(\"ignore\")\n",
    "    if ridge_true:\n",
    "        model = Ridge()\n",
    "        result_str = 'For ridge model '\n",
    "    else:\n",
    "        model = Lasso()\n",
    "        result_str = 'For lasso model '\n",
    "    if std_true:\n",
    "        x_train_temp = x_train_std\n",
    "        y_train_temp = y_train_std\n",
    "        x_test_temp = x_test_std\n",
    "        y_test_temp = y_test_std\n",
    "        result_str += 'with standardization'\n",
    "    else:\n",
    "        x_train_temp = x_train\n",
    "        y_train_temp = y_train\n",
    "        x_test_temp = x_test\n",
    "        y_test_temp = y_test\n",
    "        result_str += 'without standardization'\n",
    "    alphas = np.logspace(0,8,100)\n",
    "    grid_ridge= GridSearchCV(model, param_grid = dict(alpha = alphas), cv=5, scoring='neg_mean_squared_error')\n",
    "    grid_ridge.fit(x_train_temp, y_train_temp)\n",
    "    alpha_without_ridge = grid_ridge.best_estimator_.alpha\n",
    "    result_str += \"\\nthe best alpha is %.2f and the best score is %.4f\" %(alpha_without_ridge, grid_ridge.best_score_)\n",
    "    ridge = Ridge(alpha = alpha_without_ridge)\n",
    "    ridge.fit(x_train_temp, y_train_temp)\n",
    "    RSS = np.sum((ridge.predict(x_test_temp) - y_test_temp) ** 2)\n",
    "    tr_R2 = ridge.score(x_train_temp, y_train_temp)\n",
    "    te_R2 = ridge.score(x_test_temp, y_test_temp)\n",
    "    tr_er = mean_squared_error(y_train_temp, ridge.predict(x_train_temp))\n",
    "    te_er = mean_squared_error(y_test_temp, ridge.predict(x_test_temp))\n",
    "    result_str += '\\ntraining error: ' + str(tr_er) + '\\ntesting error: ' + str(te_er) + \"\\nThe R^2 for training set:\" + str(tr_R2)\n",
    "    result_str += \"\\nThe R^2 for test set:\" + str(te_R2)\n",
    "    result_str += \"\\nRSS: %.2f\" %(RSS)\n",
    "    result1 = pd.DataFrame(ridge.coef_).transpose()\n",
    "    result1.columns = x_list\n",
    "    result1['intercept'] = ridge.intercept_ \n",
    "    result1 = result1.transpose()\n",
    "    result1.columns = ['coefficient']\n",
    "    result1\n",
    "    result_dict = {'RSS': RSS, 'training R2': tr_R2, 'testing R2': te_R2, 'training error': tr_er, 'testing R2': te_er, 'alpha': alpha_without_ridge}\n",
    "    return result_str, result1, result_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs_str, rs1, rs_dict = repeat_test(True, True)\n",
    "rn_str, rn1, rn_dict = repeat_test(True, False)\n",
    "ls_str, ls1, ls_dict = repeat_test(False, True)\n",
    "ln_str, ln1, ln_dict = repeat_test(False, False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For ridge model with standardization\n",
      "the best alpha is 1.00 and the best score is -1572313.6346\n",
      "training error: 1250765.8458008112\n",
      "testing error: 1589538.1301434354\n",
      "The R^2 for training set:0.9185173955752879\n",
      "The R^2 for test set:0.8804561448160135\n",
      "RSS: 247967948.30\n",
      "--------------------\n",
      "For ridge model without standardization\n",
      "the best alpha is 2056.51 and the best score is -1298199.1190\n",
      "training error: 1022037.3601631394\n",
      "testing error: 1444655.7529332454\n",
      "The R^2 for training set:0.9334181803852097\n",
      "The R^2 for test set:0.8913522646331353\n",
      "RSS: 225366297.46\n",
      "--------------------\n",
      "For lasso model with standardization\n",
      "the best alpha is 1.00 and the best score is -1470487.6623\n",
      "training error: 1250765.8458008112\n",
      "testing error: 1589538.1301434354\n",
      "The R^2 for training set:0.9185173955752879\n",
      "The R^2 for test set:0.8804561448160135\n",
      "RSS: 247967948.30\n",
      "--------------------\n",
      "For lasso model without standardization\n",
      "the best alpha is 220.51 and the best score is -1297819.5340\n",
      "training error: 1021140.8074146521\n",
      "testing error: 1442840.187574349\n",
      "The R^2 for training set:0.9334765873629796\n",
      "The R^2 for test set:0.8914888072414724\n",
      "RSS: 225083069.26\n"
     ]
    }
   ],
   "source": [
    "print(rs_str)\n",
    "print('-'*20)\n",
    "print(rn_str)\n",
    "print('-'*20)\n",
    "print(ls_str)\n",
    "print('-'*20)\n",
    "print(ln_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ridge_std</th>\n",
       "      <th>ridge_not_std</th>\n",
       "      <th>lasso_std</th>\n",
       "      <th>lasso_not_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Accept</th>\n",
       "      <td>5970.593651</td>\n",
       "      <td>1.661353</td>\n",
       "      <td>5970.593651</td>\n",
       "      <td>1.664409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Enroll</th>\n",
       "      <td>-1726.879874</td>\n",
       "      <td>-0.996341</td>\n",
       "      <td>-1726.879874</td>\n",
       "      <td>-1.001874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Top10perc</th>\n",
       "      <td>843.214745</td>\n",
       "      <td>47.019417</td>\n",
       "      <td>843.214745</td>\n",
       "      <td>50.525712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Top25perc</th>\n",
       "      <td>690.596803</td>\n",
       "      <td>-12.245495</td>\n",
       "      <td>690.596803</td>\n",
       "      <td>-14.661253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F.Undergrad</th>\n",
       "      <td>227.410203</td>\n",
       "      <td>0.074535</td>\n",
       "      <td>227.410203</td>\n",
       "      <td>0.073456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P.Undergrad</th>\n",
       "      <td>-433.924746</td>\n",
       "      <td>-0.017740</td>\n",
       "      <td>-433.924746</td>\n",
       "      <td>-0.015181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Outstate</th>\n",
       "      <td>-227.680310</td>\n",
       "      <td>-0.101645</td>\n",
       "      <td>-227.680310</td>\n",
       "      <td>-0.101315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Room.Board</th>\n",
       "      <td>579.110401</td>\n",
       "      <td>0.136495</td>\n",
       "      <td>579.110401</td>\n",
       "      <td>0.136372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Books</th>\n",
       "      <td>639.138318</td>\n",
       "      <td>0.097486</td>\n",
       "      <td>639.138318</td>\n",
       "      <td>0.083943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Personal</th>\n",
       "      <td>31.189998</td>\n",
       "      <td>0.046206</td>\n",
       "      <td>31.189998</td>\n",
       "      <td>0.047102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PhD</th>\n",
       "      <td>53.876322</td>\n",
       "      <td>-7.184440</td>\n",
       "      <td>53.876322</td>\n",
       "      <td>-8.070647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Terminal</th>\n",
       "      <td>9.933560</td>\n",
       "      <td>0.735641</td>\n",
       "      <td>9.933560</td>\n",
       "      <td>1.534350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S.F.Ratio</th>\n",
       "      <td>8.893351</td>\n",
       "      <td>10.292946</td>\n",
       "      <td>8.893351</td>\n",
       "      <td>14.071214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>perc.alumni</th>\n",
       "      <td>78.965003</td>\n",
       "      <td>-2.329414</td>\n",
       "      <td>78.965003</td>\n",
       "      <td>-2.400135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Expend</th>\n",
       "      <td>362.302746</td>\n",
       "      <td>0.047482</td>\n",
       "      <td>362.302746</td>\n",
       "      <td>0.046231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Grad.Rate</th>\n",
       "      <td>437.451168</td>\n",
       "      <td>7.789496</td>\n",
       "      <td>437.451168</td>\n",
       "      <td>7.888382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>intercept</th>\n",
       "      <td>-2.107159</td>\n",
       "      <td>-690.194625</td>\n",
       "      <td>-2.107159</td>\n",
       "      <td>-693.480395</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               ridge_std  ridge_not_std    lasso_std  lasso_not_std\n",
       "Accept       5970.593651       1.661353  5970.593651       1.664409\n",
       "Enroll      -1726.879874      -0.996341 -1726.879874      -1.001874\n",
       "Top10perc     843.214745      47.019417   843.214745      50.525712\n",
       "Top25perc     690.596803     -12.245495   690.596803     -14.661253\n",
       "F.Undergrad   227.410203       0.074535   227.410203       0.073456\n",
       "P.Undergrad  -433.924746      -0.017740  -433.924746      -0.015181\n",
       "Outstate     -227.680310      -0.101645  -227.680310      -0.101315\n",
       "Room.Board    579.110401       0.136495   579.110401       0.136372\n",
       "Books         639.138318       0.097486   639.138318       0.083943\n",
       "Personal       31.189998       0.046206    31.189998       0.047102\n",
       "PhD            53.876322      -7.184440    53.876322      -8.070647\n",
       "Terminal        9.933560       0.735641     9.933560       1.534350\n",
       "S.F.Ratio       8.893351      10.292946     8.893351      14.071214\n",
       "perc.alumni    78.965003      -2.329414    78.965003      -2.400135\n",
       "Expend        362.302746       0.047482   362.302746       0.046231\n",
       "Grad.Rate     437.451168       7.789496   437.451168       7.888382\n",
       "intercept      -2.107159    -690.194625    -2.107159    -693.480395"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coefficient = pd.concat([rs1, rn1, ls1, ln1], axis=1)\n",
    "rename_columns = ['ridge_std', 'ridge_not_std', 'lasso_std', 'lasso_not_std']\n",
    "coefficient.columns = rename_columns\n",
    "coefficient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RSS</th>\n",
       "      <th>training R2</th>\n",
       "      <th>testing R2</th>\n",
       "      <th>training error</th>\n",
       "      <th>alpha</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.479679e+08</td>\n",
       "      <td>0.918517</td>\n",
       "      <td>1.589538e+06</td>\n",
       "      <td>1.250766e+06</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>ridge_std</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.253663e+08</td>\n",
       "      <td>0.933418</td>\n",
       "      <td>1.444656e+06</td>\n",
       "      <td>1.022037e+06</td>\n",
       "      <td>2056.512308</td>\n",
       "      <td>ridge_not_std</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.479679e+08</td>\n",
       "      <td>0.918517</td>\n",
       "      <td>1.589538e+06</td>\n",
       "      <td>1.250766e+06</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>lasso_std</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.253663e+08</td>\n",
       "      <td>0.933418</td>\n",
       "      <td>1.444656e+06</td>\n",
       "      <td>1.022037e+06</td>\n",
       "      <td>2056.512308</td>\n",
       "      <td>lasso_not_std</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            RSS  training R2    testing R2  training error        alpha  \\\n",
       "0  2.479679e+08     0.918517  1.589538e+06    1.250766e+06     1.000000   \n",
       "1  2.253663e+08     0.933418  1.444656e+06    1.022037e+06  2056.512308   \n",
       "2  2.479679e+08     0.918517  1.589538e+06    1.250766e+06     1.000000   \n",
       "3  2.253663e+08     0.933418  1.444656e+06    1.022037e+06  2056.512308   \n",
       "\n",
       "            type  \n",
       "0      ridge_std  \n",
       "1  ridge_not_std  \n",
       "2      lasso_std  \n",
       "3  lasso_not_std  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "factors = pd.DataFrame.from_dict([rs_dict, rn_dict, ls_dict, rn_dict])\n",
    "factors['type'] = rename_columns\n",
    "factors.set_index('type')\n",
    "factors"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
